# -*- coding: UTF-8 -*-
import pandas as pd
import modin.pandas as mpd

import time,os,datetime,math,struct,sys
from colorama import init, Fore, Back, Style
# from makedir import FileOp
pd.set_option('display.expand_frame_repr', False)
init(autoreset=True)
import logging
from loguru import logger
from struct import *
# from tqdm import tqdm
currentDate = time.strftime("%Y%m%d", time.localtime())
LOG_DIR = os.path.expanduser("logs/")
LOG_FILE = os.path.join(LOG_DIR, currentDate+"read_big_csv.log")
logger.add(LOG_FILE,rotation = "2000KB" , encoding='gb2312')   #压缩成zip格式
# import ray
class FileOp:
    def mkdir(path):
        # 引入模块
        import os
        # 去除首位空格
        path = path.strip()
        # 去除尾部 \ 符号
        path = path.rstrip("\\")
        # 判断路径是否存在# 存在     True# 不存在   False
        isExists = os.path.exists(path)
        # 判断结果
        if not isExists:
            # 如果不存在则创建目录
            # 创建目录操作函数
            os.makedirs(path)
            logger.success(path + ' 创建成功')
            return True
        else:
            # 如果目录存在则不创建，并提示目录已存在
            logger.warning(path + ' 目录已存在')
            return False

    @staticmethod
    def get_totalsize(path_add):

        size = 0
        # 遍历并装载第一层的文件和文件夹
        list = os.listdir(path_add)

        for i in list:
            # 拼接成完整的绝对路径
            pathfile = os.path.join(path_add, i)
            # 文件 OR NOT
            if os.path.isfile(pathfile):
                # print(i," is FILE")
                # print(pathfile)
                # 统计文件大小
                size += os.path.getsize(pathfile)
            # 文件夹 OR NOT
            elif os.path.isdir(pathfile):
                # print(i," is FOLDER")
                # print(pathfile)
                # 递归统计文件夹里面的文件名称
                size += FileOp.get_totalsize(pathfile)

        return size


    @staticmethod
    def deldir(location):
        import glob
        # Importing os and glob modules
        # Loop Through the folder projects all files and deleting them one by one
        for file in glob.glob(location + "/*"):
            os.remove(file)
            logger.success("Deleted " + str(file))
    # result = getTotalsize(path_add)
    # print("Total size:{}字节({}MB)".format(result, result // 1024 // 1024))  # 输出总大小
#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
import configparser
class Config:
    def __init__(self,filename,encoding="gbk"):
        logger.info("Config:def __init__(self):初始化"+filename+encoding)
        self.cf = configparser.ConfigParser()
        self.cf.read(filename, encoding=encoding)  # 读取配置文件，如果写文件的绝对路径，就可以不用os模块
    def get_config(self,rootstr,txt):
        return self.cf.get(rootstr, txt)

@logger.catch
def show_title(title,funcxxx, **kwargs):
    print("-" * 150)
    logger.info('show_title')

    starttime = datetime.datetime.now()

    funcxxx(title,**kwargs)

    endtime = datetime.datetime.now()
    logger.success('程序运行时间:'+ str(endtime - starttime))

@logger.catch
def set_running(title):
    logger.info('set_running')
    cf = Config("config.ini")
    currentDate = time.strftime("%Y%m%d", time.localtime())
    # currentDate = currentDate + "_final"
    targetDir = cf.get_config("IOFileConfig", "exportdir")
    logger.success(targetDir)
    dir_list=os.listdir(targetDir)
    dirfilecount=len(dir_list)
    logger.success("文件数量:"+str(dirfilecount))
    expdir="E:\\tdx_export\\hdf\\"
    FileOp.mkdir(expdir)
    logger.success(expdir)
    # logger.success(str(type(dir_list))+str(dirfilecount))
    # pbar = tqdm(dir_list,colour='yellow')
    # for char in pbar:
    #     pbar.set_description("Progress %s" %char)
    scale=dirfilecount
    LL=40
    color_para=Fore.RED
    i=1
    df_count_totle=0
    g_sum=0
    logger.info("开始循环。。。")
    print("-"*150)
    tiaojian_count = 1
    for __list in dir_list:

        filepath = targetDir + __list
        df = pd.read_csv(filepath)

        msg = ""  # getattr(funcxxx, '__name__')
        # print("csvfilepath+__listcsvfilepath+__list",csvfilepath+__list)
        # msg1, msg2 = funcxxx(__list, **kwargs)


        # print(df.shape[0])
        if i==1:
            tmpdf=df
        else:
            xdf = pd.concat([tmpdf, df], axis=0, ignore_index=True)
            tmpdf =xdf

        df_count_totle +=df.shape[0]


        a = (i / scale) * 100
        b = '.' * LL
        c = '▋' * (int((i * LL) / scale))
        b = '.' * ((LL - 1) - int((i * LL) / scale))
        dur = time.perf_counter()  # //每次获取当前时间
        # print(__list)
        print(Style.BRIGHT + Fore.RED + "\r[{}]{:^3.0f}% {}{} {:.2f}s".format(i, a, c, b, dur),__list,end=' ')
        # print(filepath, "[" + str(df.shape[0]) + "]")
        # time.sleep(0.01)
        N = scale
        if i >= N:
            # df.to_csv("big.csv")
            df.to_hdf(expdir+"\\"+str(tiaojian_count)+"big.h5", 'df', mode='w')
            g_sum += df.shape[0]
            print(tiaojian_count,expdir+"\\"+str(tiaojian_count)+"big.h5", "[" + str(g_sum) + "]", df_count_totle,tmpdf.shape[0])
            break
        else:
            # print(Fore.RED + str(int(N % 3)))
            g_sum += df.shape[0]

            if i % 2000 == 0:

                df.to_hdf(expdir+"\\"+str(tiaojian_count)+"big.h5", 'df', mode='w')
                print(tiaojian_count, expdir+"\\"+str(tiaojian_count)+"big.h5", "[" + str(g_sum) + "]", df_count_totle,"DF:",tmpdf.shape[0])
                tmpdf.drop(tmpdf.index, inplace=True)
                tiaojian_count += 1
                # tmpdf=df
                g_sum = 0
            # else:
            #     # tiaojian_count += 1



        i += 1

    print("-" * 150)
    logger.critical("循环"+str(i)+",记录总数:"+str(df_count_totle))
    # print(char)
    # ray.init()
    # df=pd.read_hdf("big.h5")
    # print(df.shape[0])
if __name__ == '__main__':
    os.system("cls")
    show_title("title",funcxxx=set_running)

